{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "X6-q-gTUaZU7",
    "tags": []
   },
   "source": [
    "# Lesson 2: RAG Triad of metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 47
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 98
   },
   "outputs": [],
   "source": [
    "import utils\n",
    "\n",
    "import os\n",
    "import openai\n",
    "openai.api_key = utils.get_openai_api_key()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 81,
    "id": "IBfdyn3MaZU9"
   },
   "outputs": [],
   "source": [
    "from trulens_eval import Tru\n",
    "\n",
    "tru = Tru()\n",
    "tru.reset_database()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 98,
    "id": "wMvq1q8yaZU-"
   },
   "outputs": [],
   "source": [
    "from llama_index import SimpleDirectoryReader\n",
    "\n",
    "documents = SimpleDirectoryReader(\n",
    "    input_files=[\"./eBook-How-to-Build-a-Career-in-AI.pdf\"]\n",
    ").load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 81,
    "id": "sY8Oui4taZU-"
   },
   "outputs": [],
   "source": [
    "from llama_index import Document\n",
    "\n",
    "document = Document(text=\"\\n\\n\".\\\n",
    "                    join([doc.text for doc in documents]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 217
   },
   "outputs": [],
   "source": [
    "from utils import build_sentence_window_index\n",
    "\n",
    "from llama_index.llms import OpenAI\n",
    "\n",
    "llm = OpenAI(model=\"gpt-3.5-turbo\", temperature=0.1)\n",
    "\n",
    "sentence_index = build_sentence_window_index(\n",
    "    document,\n",
    "    llm,\n",
    "    embed_model=\"local:BAAI/bge-small-en-v1.5\",\n",
    "    save_dir=\"sentence_index\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 81
   },
   "outputs": [],
   "source": [
    "from utils import get_sentence_window_query_engine\n",
    "\n",
    "sentence_window_engine = \\\n",
    "get_sentence_window_query_engine(sentence_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 64
   },
   "outputs": [],
   "source": [
    "output = sentence_window_engine.query(\n",
    "    \"How do you create your AI portfolio?\")\n",
    "output.response"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feedback functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 64,
    "id": "5KqV-IbQaZVB"
   },
   "outputs": [],
   "source": [
    "import nest_asyncio\n",
    "\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 64
   },
   "outputs": [],
   "source": [
    "from trulens_eval import OpenAI as fOpenAI\n",
    "\n",
    "provider = fOpenAI()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Answer Relevance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 115
   },
   "outputs": [],
   "source": [
    "from trulens_eval import Feedback\n",
    "\n",
    "f_qa_relevance = Feedback(\n",
    "    provider.relevance_with_cot_reasons,\n",
    "    name=\"Answer Relevance\"\n",
    ").on_input_output()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Context Relevance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 64
   },
   "outputs": [],
   "source": [
    "from trulens_eval import TruLlama\n",
    "\n",
    "context_selection = TruLlama.select_source_nodes().node.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 166
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "f_qs_relevance = (\n",
    "    Feedback(provider.qs_relevance,\n",
    "             name=\"Context Relevance\")\n",
    "    .on_input()\n",
    "    .on(context_selection)\n",
    "    .aggregate(np.mean)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 166
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "f_qs_relevance = (\n",
    "    Feedback(provider.qs_relevance_with_cot_reasons,\n",
    "             name=\"Context Relevance\")\n",
    "    .on_input()\n",
    "    .on(context_selection)\n",
    "    .aggregate(np.mean)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Groundedness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 64
   },
   "outputs": [],
   "source": [
    "from trulens_eval.feedback import Groundedness\n",
    "\n",
    "grounded = Groundedness(groundedness_provider=provider)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 149,
    "id": "kXJBD4gfaZVC"
   },
   "outputs": [],
   "source": [
    "f_groundedness = (\n",
    "    Feedback(grounded.groundedness_measure_with_cot_reasons,\n",
    "             name=\"Groundedness\"\n",
    "            )\n",
    "    .on(context_selection)\n",
    "    .on_output()\n",
    "    .aggregate(grounded.grounded_statements_aggregator)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation of the RAG application"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 217,
    "id": "KUDHInR-aZVC"
   },
   "outputs": [],
   "source": [
    "from trulens_eval import TruLlama\n",
    "from trulens_eval import FeedbackMode\n",
    "\n",
    "tru_recorder = TruLlama(\n",
    "    sentence_window_engine,\n",
    "    app_id=\"App_1\",\n",
    "    feedbacks=[\n",
    "        f_qa_relevance,\n",
    "        f_qs_relevance,\n",
    "        f_groundedness\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 115,
    "id": "dsA3ziw1aZVD"
   },
   "outputs": [],
   "source": [
    "eval_questions = []\n",
    "with open('eval_questions.txt', 'r') as file:\n",
    "    for line in file:\n",
    "        # Remove newline character and convert to integer\n",
    "        item = line.strip()\n",
    "        eval_questions.append(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "eval_questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "eval_questions.append(\"How can I be successful in AI?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "eval_questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 64,
    "id": "01_P6TxaaZVD"
   },
   "outputs": [],
   "source": [
    "for question in eval_questions:\n",
    "    with tru_recorder as recording:\n",
    "        sentence_window_engine.query(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 47,
    "id": "sNPhDde6ZArq"
   },
   "outputs": [],
   "source": [
    "records, feedback = tru.get_records_and_feedback(app_ids=[])\n",
    "records.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 81
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "pd.set_option(\"display.max_colwidth\", None)\n",
    "records[[\"input\", \"output\"] + feedback]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 30
   },
   "outputs": [],
   "source": [
    "tru.get_leaderboard(app_ids=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "height": 30,
    "id": "6Yp4_e4faZVD"
   },
   "outputs": [],
   "source": [
    "tru.run_dashboard()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": [
    {
     "file_id": "1ChKW1kEIUcUVDDTWjpA5Tf_ib3Hhp3uS",
     "timestamp": 1695164681916
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
